import OutputDataParser;
import time;


def detag(text):
    text.find('<');

path = '/mounts/timan3/disks/0/projects/blogcrawler/testplace/blog/crawler/src/081021/posting.dat';
outpath = '090101.090901.posting.dat';
pos = 191650000000;
format =  '%Y/%m/%d';
minDate = time.mktime(time.strptime('2009/01/01', format));
maxDate = time.mktime(time.strptime('2009/09/01', format));

t0 = time.time();
file = OutputDataParser.PostingFile();
file.open(path);
file.seek(pos);
out = open(outpath, 'w');
cnt = 1;
effcnt = 1;
dateCnts = {};
while file.next() >= 0:
    posting = file.getCurrElement();
    dateStr = time.strftime(format, time.gmtime(posting.date));
    if posting.date <= maxDate and posting.date >= minDate and len(posting.description) > 200 and (not dateCnts.__contains__(dateStr) or dateCnts[dateStr] < 10000):
        posting.description = detag(posting.description);
        if(len(posting.description) > 100):
            out.write(posting.__str__());
            out.write('\n');
            effcnt += 1;
        if dateCnts.__contains__(dateStr):
            dateCnts[dateStr] += 1;
        else:
            dateCnts[dateStr] = 1;
    cnt += 1;
    if cnt % 5000 == 0:
        print cnt, effcnt, dateStr, int(time.time() - t0);    
out.close();
